{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from core import Agent\n",
    "import gym\n",
    "from puckworld import PuckWorldEnv\n",
    "from mountain_car import MountainCarEnv\n",
    "from utils import learning_curve\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LinearSarsaLambdaAgent(Agent):\n",
    "    '''仅适用状态为连续，行为为离散的情况'''\n",
    "    def __init__(self, env):\n",
    "        super(LinearSarsaLambdaAgent, self).__init__(env)\n",
    "        # 特征由3部分组成：状态、行为、一个偏置项b\n",
    "        # 如果状态或行为是离散的，状态的特征数为离散的总数\n",
    "        self.n_state = self.obs_space.low.size\n",
    "        self.n_action = self.action_space.n\n",
    "        self.state_range = self.obs_space.high - self.obs_space.low\n",
    "        self.n_feature = self.n_state + self.n_action + 1 # 包括一个偏置项\n",
    "        \n",
    "        self.w = np.zeros((self.n_feature)) # parameters\n",
    "\n",
    "        \n",
    "    def get_array(self, value, n = None):\n",
    "        '''将一个索引代表的数据转化为one-hot向量\n",
    "        '''\n",
    "        if type(value) in [list, tuple, np.ndarray]:\n",
    "            return value\n",
    "        result = np.zeros(n)\n",
    "        result[value] = 1\n",
    "        return result\n",
    "    \n",
    "    def normalized_state(self, state):\n",
    "        return self.get_array(state, self.n_state)/self.state_range\n",
    "    \n",
    "    def get_features(self, state, action):\n",
    "        state = self.normalized_state(state)\n",
    "        action = self.get_array(action, self.n_action)\n",
    "        return np.hstack([state, action, [1]])\n",
    "    \n",
    "    def get_Q(self, state, action):\n",
    "        # 均一化\n",
    "        features = self.get_features(state, action)\n",
    "        return np.dot(self.w, features)\n",
    "    \n",
    "    def get_max_Q(self, state):\n",
    "        max_q = -float('inf')\n",
    "        for a_opt in range(self.n_action):\n",
    "            q = self.get_Q(state, a_opt)\n",
    "            if q >= max_q:\n",
    "                max_q = q\n",
    "        return max_q\n",
    "        \n",
    "    def greedy_policy(self, state):\n",
    "        max_q, a_max_q = -float('inf'), []\n",
    "        for a_opt in range(self.n_action):\n",
    "            q = self.get_Q(state, a_opt)\n",
    "            #print(q)\n",
    "            if q > max_q:\n",
    "                max_q = q\n",
    "                a_max_q = [a_opt]\n",
    "            elif q == max_q:\n",
    "                a_max_q.append(a_opt)\n",
    "        if a_max_q == []:\n",
    "            print(\"no action to be choice\")\n",
    "        return random.choice(a_max_q)\n",
    "    \n",
    "    def epsilon_greedy_policy(self, state, epsilon = 0.05):\n",
    "        rand_value = random.random()\n",
    "        if rand_value < epsilon:\n",
    "            return random.choice([i for i in range(self.n_action)])\n",
    "        else:\n",
    "            return self.greedy_policy(state)\n",
    "        \n",
    "    def perform_policy(self, state, epsilon = 0.05):\n",
    "        return self.epsilon_greedy_policy(state, epsilon)\n",
    "    \n",
    "    def learning_method(self,lambda_ = 0.9, gamma = 0.99, alpha = 0.02, epsilon = 0.2, display = False):\n",
    "        self.state = self.env.reset()\n",
    "        s0 = self.state\n",
    "        if display:\n",
    "            self.env.render()\n",
    "        time_in_episode, total_reward = 0, 0\n",
    "        is_done = False\n",
    "        loss = 0\n",
    "        E = np.zeros((self.n_feature)) # reset traces\n",
    "        while not is_done:\n",
    "            # add code here\n",
    "            s0 = self.state\n",
    "            a0 = self.perform_policy(s0, epsilon)\n",
    "            s1, r1, is_done, info, total_reward = self.act(a0)\n",
    "            if display:\n",
    "                self.env.render()\n",
    "                \n",
    "            old_Q = self.get_Q(s0, a0)\n",
    "            td_error = r1 - old_Q\n",
    "            E[a0] += 1\n",
    "            if not is_done:\n",
    "                td_error += gamma * self.get_max_Q(s1)\n",
    "            \n",
    "            gradient = self.get_features(s0, a0)\n",
    "            loss += td_error * td_error\n",
    "            delta_w = alpha * td_error * E\n",
    "            self.w += delta_w\n",
    "            #print(\"gradient:{}\".format(gradient))           \n",
    "            #print(\"weight:{}\".format(self.w))\n",
    "            E = gamma * lambda_ * E + gradient\n",
    "             \n",
    "            time_in_episode += 1\n",
    "        loss /= 2*time_in_episode   \n",
    "        if display:\n",
    "            print(\"epsilon:{:3.2f},loss:{:3.2f},{}\".format(epsilon,loss,self.experience.last_episode))\n",
    "        return time_in_episode, total_reward  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "env = MountainCarEnv()\n",
    "env.reset()\n",
    "agent = LinearSarsaLambdaAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.8000001, 0.14     ], dtype=float32)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state = env.reset()\n",
    "agent.normalized_state(state)\n",
    "agent.state_range"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|▍         | 8/200 [00:27<11:04,  3.46s/it]/home/qiang/venv/lib/python3.5/site-packages/ipykernel_launcher.py:93: RuntimeWarning: overflow encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no action to be choice\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "Cannot choose from an empty sequence",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m~/venv/lib/python3.5/random.py\u001b[0m in \u001b[0;36mchoice\u001b[0;34m(self, seq)\u001b[0m\n\u001b[1;32m    252\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 253\u001b[0;31m             \u001b[0mi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_randbelow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    254\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venv/lib/python3.5/random.py\u001b[0m in \u001b[0;36m_randbelow\u001b[0;34m(self, n, int, maxsize, type, Method, BuiltinMethod)\u001b[0m\n\u001b[1;32m    229\u001b[0m             \u001b[0mk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbit_length\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# don't use (n-1) here because n can be 1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 230\u001b[0;31m             \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetrandbits\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m          \u001b[0;31m# 0 <= r < 2**k\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    231\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0mr\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: number of bits must be greater than zero",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-51b378fcf769>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m                \u001b[0mmin_epsilon\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1e-3\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m                \u001b[0mmin_epsilon_ratio\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.8\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m                display = False)\n\u001b[0m",
      "\u001b[0;32m~/Documents/Codes/c06/core.py\u001b[0m in \u001b[0;36mlearning\u001b[0;34m(self, lambda_, epsilon, decaying_epsilon, gamma, alpha, max_episode_num, display, min_epsilon, min_epsilon_ratio)\u001b[0m\n\u001b[1;32m    274\u001b[0m                                                 target_episode = int(max_episode_num * min_epsilon_ratio))\n\u001b[1;32m    275\u001b[0m             time_in_episode, episode_reward = self.learning_method(lambda_ = lambda_, \\\n\u001b[0;32m--> 276\u001b[0;31m                   gamma = gamma, alpha = alpha, epsilon = epsilon, display = display)\n\u001b[0m\u001b[1;32m    277\u001b[0m             \u001b[0mtotal_time\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mtime_in_episode\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    278\u001b[0m             \u001b[0mnum_episode\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-2-9058171a0f78>\u001b[0m in \u001b[0;36mlearning_method\u001b[0;34m(self, lambda_, gamma, alpha, epsilon, display)\u001b[0m\n\u001b[1;32m     79\u001b[0m             \u001b[0;31m# add code here\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m             \u001b[0ms0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 81\u001b[0;31m             \u001b[0ma0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mperform_policy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepsilon\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     82\u001b[0m             \u001b[0ms1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mr1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mis_done\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minfo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal_reward\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mact\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     83\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mdisplay\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-2-9058171a0f78>\u001b[0m in \u001b[0;36mperform_policy\u001b[0;34m(self, state, epsilon)\u001b[0m\n\u001b[1;32m     65\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mperform_policy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepsilon\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.05\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 67\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepsilon_greedy_policy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepsilon\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     68\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mlearning_method\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlambda_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.9\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgamma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.99\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malpha\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.02\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepsilon\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdisplay\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-2-9058171a0f78>\u001b[0m in \u001b[0;36mepsilon_greedy_policy\u001b[0;34m(self, state, epsilon)\u001b[0m\n\u001b[1;32m     62\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchoice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_action\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgreedy_policy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mperform_policy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepsilon\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.05\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-2-9058171a0f78>\u001b[0m in \u001b[0;36mgreedy_policy\u001b[0;34m(self, state)\u001b[0m\n\u001b[1;32m     55\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0ma_max_q\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"no action to be choice\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 57\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchoice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma_max_q\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     58\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mepsilon_greedy_policy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepsilon\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.05\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venv/lib/python3.5/random.py\u001b[0m in \u001b[0;36mchoice\u001b[0;34m(self, seq)\u001b[0m\n\u001b[1;32m    253\u001b[0m             \u001b[0mi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_randbelow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    254\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 255\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mIndexError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Cannot choose from an empty sequence'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    256\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mseq\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    257\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: Cannot choose from an empty sequence"
     ]
    }
   ],
   "source": [
    "data = agent.learning(gamma=0.999,          # 衰减引子\n",
    "               epsilon = 1,\n",
    "               decaying_epsilon = True,\n",
    "               alpha = 1e-4, \n",
    "               max_episode_num = 200, \n",
    "               min_epsilon = 1e-3,\n",
    "               min_epsilon_ratio = 0.8,\n",
    "               display = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(agent.w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_curve(data, 2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = agent.learning(gamma=0.99,          # 衰减引子\n",
    "               epsilon = 0.001,\n",
    "               decaying_epsilon = False,\n",
    "               alpha = 1e-5, \n",
    "               max_episode_num = 100, \n",
    "               display = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
